# -*- coding:utf-8 -*-

import os
import sys
import thread
import threading
import time
import shutil
import ftplib

from MyException import Error, NoKeyError, RecycleError, RetryError, OpenFileError
from MyLogger import MyLogger, TIMEROTATE, SIZEROTATE
from MutexList import MutexList
from StatLog import StatLog

class FTPConnect:
    def __init__(self, ip, port, username, password, tmpdir, dstdir):
        """
        初始化FTP接口信息
        """
        self.ip = ip
        self.port = port
        self.username = username
        self.password = password
        self.tmp_dir = tmpdir
        self.dst_dir = dstdir
        self.ftp = ftplib.FTP()
        
    def show(self):
        """
        打印FTP接口信息
        """
        print "HOST:", self.ip
        print "PORT:", self.port
        print "USERNAME:", self.username
        print "PASSWORD:", self.password
        print "TMP_DIR:", self.tmp_dir
        print "DST_DIR:", self.dst_dir
        
    def check_ftp_connect(self):
        """
        检测FTP服务是否可连接
        @return: True-FTP服务可连接，False-FTP服务不可连接
        """
        try:
            self.ftp.connect(self.ip, self.port)
            self.ftp.login(self.username, self.password)
            return True
        except:
            return False
    
    def check_ftp_used(self, file):
        """
        通过写临时文件，检测FTP服务是否可用
        @return: 0-不可连接，1-不可登陆，2-写临时文件失败，3-重命名失败，
                4-删除临时文件失败， 5-FTP服务可用， -1-未知错误
        """
        try:
            #拼接文件名    
            filename = os.path.basename(file)
            tmpname = os.path.join(self.tmp_dir, filename + '.temp')
            dstname = os.path.join(self.dst_dir, filename)
            
            #连接FTP服务器
            try:
                self.ftp.connect(self.ip, self.port)
            except:
                return 0
            
            #登陆FTP服务器
            try:
                self.ftp.login(self.username, self.password)
            except:
                return 1
                
            #写临时文件
            try:
                fp = open(file, 'r')
                buf_size = 1024
                ftpcmd = "APPE " + tmpname
                self.ftp.storbinary(ftpcmd, fp, buf_size)
                fp.close()
            except:
                return 2
            
            #重命名
            try:   
                self.ftp.rename(tmpname, dstname)
            except:
                return 3
            
            #删除文件
            try:
                self.ftp.delete(dstname)
            except:
                return 4
            
            return 5
        
        except:
            return -1
        
    def disconnect(self):
        """
        断开FTP连接
        """
        try:
            self.ftp.close()
        except:
            pass
        
class FTPPut(threading.Thread):
    def __init__(self, ftp_name, send_dir, backup_dir, valid_ftp_list, invalid_ftp_list, receive_file_list, 
                 send_file_list, is_write_statlog, statinfo_map, statlog, data_dest, rename_func_list,
                 success_logger, error_logger):
        """
        初始化
        """
        #初始化线程类
        threading.Thread.__init__(self)
        #初始化化FTP名
        self.ftp_name = ftp_name
        #初始化发送目录和备份目录
        self.send_dir = send_dir
        self.backup_dir = backup_dir
        #初始化FTP连接列表
        self.valid_ftp_list = valid_ftp_list
        self.invalid_ftp_list = invalid_ftp_list
        #初始化文件列表
        self.receive_file_list = receive_file_list
        self.send_file_list = send_file_list
        #初始化程序日志
        self.success_logger = success_logger
        self.error_logger = error_logger
        #初始化统计日志开关
        self.is_write_statlog = is_write_statlog
        #初始化统计日志信息字典
        self.statinfo_map = statinfo_map
        #初始化统计日志
        self.statlog = statlog
        #初始化数据流向
        self.data_dest = data_dest
        #初始化重命名功能列表
        self.rename_func_list = rename_func_list

    def run(self):
        """
        FTP发送线程主体
        """
        while(True):
            try:
                try:
                    #获取FTP连接
                    ftp_connect = self.valid_ftp_list.get()
                    if not ftp_connect:
                        raise RecycleError("no ftp")
                        
                    #判断FTP连接是否正常
                    try:
                        ftp_connect.ftp.pwd()
                    except Exception, msg:
                        self.invalid_ftp_list.add(ftp_connect)
                        raise RecycleError(msg)
                        
                    #获取文件
                    filename = self.receive_file_list.get()
                    if not filename:
                        #没有文件需要发送FTP连接返回可使用列表
                        self.valid_ftp_list.add(ftp_connect)
                        raise RecycleError("no file")
    
                    #获取统计日志信息
                    statinfo = self.statinfo_map.find(filename)
                    
                    #重命名文件
                    tmp_filename = filename
                    for rename in self.rename_func_list:
                        #print rename[0], rename[1]
                        tmp_filename = tmp_filename.replace(rename[0], rename[1])
                    #print tmp_filename
                    #拼接文件名
                    fullname = os.path.join(self.send_dir, filename)
                    ftp_tmpname = os.path.join(ftp_connect.tmp_dir, tmp_filename + ".temp")
                    ftp_dstname = os.path.join(ftp_connect.dst_dir, tmp_filename)
                                          
                    #支持断点续传
                    remote_size = 0
                    try:
                        remote_size = ftp_connect.ftp.size(ftp_tmpname)
                        if remote_size == None:
                            remote_size = 0
                    except:
                        pass
                        
                    #打开文件
                    try:
                        fp = open(fullname, "r")
                        fp.seek(remote_size)
                    except Exception, msg:
                        raise OpenFileError(msg)
                        
                    #发送文件
                    try:
                        #初始化开始发送时间
                        starttime = time.time()
                        #发送文件
                        buf_size = 1024
                        ftp_cmd = "APPE " + ftp_tmpname
                        ftp_connect.ftp.storbinary(ftp_cmd, fp, buf_size)
                        fp.close()
                        ftp_connect.ftp.rename(ftp_tmpname, ftp_dstname)
                        #计算传输率
                        usedtime = time.time() - starttime + 0.000001
                        rate = float(statinfo.file_size)/1024/usedtime
                        #文件加入已处理列表
                        self.send_file_list.add(filename)
                        #FTP连接加入可使用列表
                        self.valid_ftp_list.add(ftp_connect)
                        #记录程序日志
                        self.success_logger.info("Success ftp file: %s size: %s record: %s to server: %s rate: %f KB/S"%(filename, statinfo.file_size, statinfo.record_num, ftp_connect.ip, rate))
                        #记录统计日志
                        if self.is_write_statlog:
                            self.statlog.write_statlog_amply(filename, statinfo, 0, self.ftp_name, self.data_dest)
                    except Exception, msg:
                        #文件返回待处理文件列表
                        self.receive_file_list.add(filename)
                        #FTP连接加入不可使用列表
                        self.invalid_ftp_list.add(ftp_connect)                        
                        raise RetryError(msg)
                
                except RecycleError, msg:
                    time.sleep(2)
                except RetryError, msg:
                    self.error_logger.exception("ftp error")
                except NoKeyError, msg:
                    self.error_logger.exception("Error ftp file: %s size: 0 record: 0 cause: StatLog"%(filename))
                    if self.is_write_statlog:
                        self.statlog.write_statlog_curtly(filename, 0, "--", -1, self.ftp_name, self.data_dest)
                    raise
                except OpenFileError, msg:
                    self.error_logger.info("Error ftp file: %s size: %s record: %s cause: OpenFile\n%s"%(filename, statinfo.file_size, statinfo.record_num, msg))
                    if self.is_write_statlog:
                        self.write_statlog_amply(self.statlog, statinfo, -1, self.ftp_name, self.data_dest)
                    raise 
            except Error, msg:
                #文件加入已处理列表
                self.send_file_list.add(filename)
                #FTP连接加可使用列表
                self.valid_ftp_list.add(ftp_connect)
            except:
                #记录未知异常
                self.error_logger.exception("unknown error")
                raise
            
class FTPClient(threading.Thread):
    def __init__(self, profile, adv_conf_list, ftpinfo, statinfo_map):
        """
        初始化
        """
        #初始化线程基类
        threading.Thread.__init__(self)
        #初始化FTP接口信息
        self.name = ftpinfo.name
        self.ftpinfo = ftpinfo
        #初始化FTP连接列表
        self.valid_ftp_list = MutexList()
        self.invalid_ftp_list = MutexList()
        #初始化文件列表
        self.receive_file_list = MutexList()
        self.send_file_list = MutexList()
        #初始化发送目录
        self.send_dir = profile.send_dir
        #初始化文件队列控制信息
        self.max_file_num = profile.default_max_file_num  
        self.max_node_num = profile.default_max_node_num
        self.max_over_num = profile.default_max_over_num
        #初始化统计日志开关
        self.is_write_statlog = False
        #初始化统计日志字段
        self.statinfo_map = statinfo_map
        #初始化数据流向
        self.data_dest = ftpinfo.name
        #初始化重命名功能列表
        self.rename_func_list = []
        #初始化程序日志
        self.success_logger = MyLogger(os.path.join(profile.log_dir, ftpinfo.name + "_success.log"), SIZEROTATE, profile.size_rotate_backup_num)
        self.error_logger = MyLogger(os.path.join(profile.log_dir, ftpinfo.name + "_error.log"), SIZEROTATE, profile.size_rotate_backup_num)
        self.status_logger = MyLogger(os.path.join(profile.log_dir, ftpinfo.name + "_status.log"), SIZEROTATE, profile.size_rotate_backup_num)
        #初始化高级应用
        #print "---->", len(adv_conf_list), "<----"
        for adv_conf_name in ftpinfo.adv_conf:
            #print adv_conf_name
            if adv_conf_name.find("_RENAME_") > -1:
                for adv_conf in adv_conf_list:
                    #print adv_conf_name, adv_conf.app_name
                    if adv_conf.app_name == adv_conf_name:
                        self.rename_func_list.append((adv_conf.old_keyword, adv_conf.new_keyword))
            elif adv_conf_name.find("_OVERSTOCK_") > -1:
                for adv_conf in adv_conf_list:
                    if adv_conf.app_name == adv_conf_name:
                        self.max_file_num = adv_conf.max_file_num
                        self.max_node_num = adv_conf.max_node_num
                        self.max_over_num = adv_conf.max_over_num
            elif adv_conf_name.startswith("_STATLOG_") > -1:
                for adv_conf in adv_conf_list:
                    if adv_conf.app_name == adv_conf_name:
                        self.is_write_statlog = profile.is_write_statlog
                        self.data_dest = adv_conf.data_dest
            else:
                pass
        #print "------->", len(self.rename_func_list),"<------"
        #初始化文件队列积压文件数记录
        self.over_stock_list = []
        for i in range(self.max_node_num):
            self.over_stock_list.append(0)
        #初始化统计日志
        self.statlog = None
        if self.is_write_statlog:
            self.statlog = StatLog(profile.statlog_dir, True)
            self.statlog.daemon = True
            self.statlog.start() 
        #创建备份目录
        self.backup_dir = os.path.join(profile.file_backup_dir, self.name)
        if not os.path.exists(self.backup_dir):
            os.makedirs(self.backup_dir)
        #初始化FTP线程列表
        self.ftp_put_list = []

    def init_ftp_thread(self):
        """
        初始化FTP连接及发送线程
        """
        #建立FTP连接
        for ip in self.ftpinfo.ip_list:
            ftp_connect = FTPConnect(ip, self.ftpinfo.port, self.ftpinfo.username, self.ftpinfo.password, 
                                     self.ftpinfo.tmp_dir, self.ftpinfo.dst_dir)
            self.invalid_ftp_list.add(ftp_connect)
        #启动FTP发送线程
        for ip in self.ftpinfo.ip_list:
            ftp_put = FTPPut(self.name, self.send_dir, self.backup_dir, self.valid_ftp_list, self.invalid_ftp_list, 
                             self.receive_file_list, self.send_file_list, self.is_write_statlog, self.statinfo_map,
                             self.statlog, self.data_dest, self.rename_func_list, self.success_logger, self.error_logger)  
            ftp_put.daemon = True
            ftp_put.start()
            self.ftp_put_list.append(ftp_put)
            
    def wtite_status(self):
        """
        记录FTP连接状态
        """
        status_str = "running ftp server list:"
        for ftp_connect in self.valid_ftp_list.getCopy():
            status_str = status_str + " " + ftp_connect.ip 
        self.status_logger.info(status_str)
        
        status_str = "error ftp server list:"
        for ftp_connect in self.invalid_ftp_list.getCopy():
            status_str = status_str + " " + ftp_connect.ip 
        self.status_logger.info(status_str)            
    
    def check_put_thread(self):
        """
        检查FTP发送线程是否正常，只要有一个发送线程正常就认为正常
        """
        for ftp_put in self.ftp_put_list:
            if not ftp_put.isAlive():
                return False
        return True
                  
    def check_ftp(self):
        """
        检测不可用FTP服务是否恢复
        """
        #创建测试文件
        file = "../data/ftp_test.txt"
        if not os.path.exists(file):
            f = open(file, "w")
            f.write("OK")
            f.close()
        
        #检测不可用FTP服务是否恢复
        invalid_ftp_list = self.invalid_ftp_list.get_all()
        for ftp_connect in invalid_ftp_list:
            #ftp_connect.show()
            if ftp_connect.check_ftp_used(file) > 3:  #FTP服务可用
                self.valid_ftp_list.add(ftp_connect)
                self.status_logger.info("connect to %s success"%(ftp_connect.ip))
            else:   #FTP服务不可用
                self.invalid_ftp_list.add(ftp_connect)
                self.status_logger.info("connect to %s failed"%(ftp_connect.ip)) 
        
    def backup_all_file(self):
        """
        备份所有文件
        """
        if self.invalid_ftp_list.size() == len(self.ftp_put_list):
            try:
                #获取所有发送文件
                file_list = self.receive_file_list.get_all() 
                #创建备份目录
                sub_backup_dir = os.path.join(self.backup_dir, time.strftime("%Y%m%d%H",time.localtime()))
                if not os.path.exists(sub_backup_dir):
                    os.makedirs(sub_backup_dir)
                #备份文件
                for filename in file_list:
                    try:
                        try:
                            #获取统计日志
                            statinfo = self.statinfo_map.find(filename)
                            #文件重命名
                            tmp_filename = filename
                            for rename in self.rename_func_list:
                                tmp_filename = tmp_filename.replace(rename[0], rename[1])
                            #拷贝文件
                            src_file = os.path.join(self.send_dir, filename)
                            bak_file = os.path.join(sub_backup_dir, tmp_filename)
                            shutil.copy(src_file, bak_file)
                            #记录程序日志
                            self.error_logger.info("Success backup file: %s size: %s record: %s to dir: %s cause: NoFtp"%(filename, statinfo.file_size, statinfo.record_num, sub_backup_dir))
                            if self.is_write_statlog:
                                self.statlog.write_statlog_amply(filename, statinfo, -1, self.ftpinfo.name, self.data_dest)
                        except NoKeyError, msg:
                            self.error_logger.exception("Error backup file: %s size: 0 record: 0 cause: StatLog"%(filename))
                            if self.is_write_statlog:
                                self.statlog.write_statlog_curtly(filename, 0, "--", -1, self.ftpinfo.name, self.data_dest)
                        except Exception, msg:
                            self.error_logger.exception("Error backup file: %s size: %s record: %s cause: Unknown"%(filename, statinfo.file_size, statinfo.record_num))
                            if self.is_write_statlog:
                                self.statlog.write_statlog_amply(filename, statinfo, -1, self.ftpinfo.name, self.data_dest)
                    except:
                        self.error_logger.exception("unknown error")
                    self.send_file_list.add(filename)
            except:
                self.error_logger.exception("unknown error")
                raise
                
    def backup_stock_file(self):
        """
        备份待发送文件列表中积压文件
        """
        #更新积压文件数据列表
        over_stock_num = self.receive_file_list.size()
        self.over_stock_list.pop(0)
        self.over_stock_list.append(over_stock_num)
        #print over_stock_num, self.max_file_num
        if over_stock_num > self.max_file_num:
            #计算历史积压文件数超过最大值的次数
            over_stock_count = 0
            for over_stock in self.over_stock_list:
                if over_stock > self.max_file_num:
                    over_stock_count += 1
            #备份积压文件
            #print over_stock_count, self.max_over_num
            if over_stock_count > self.max_over_num:
                try:
                    #记录状态日志
                    self.status_logger.info("%s receive file list block. over stock: %s"%(self.name, ", ".join([str(over_stock) for over_stock in self.over_stock_list])))
                    #创建备份目录
                    sub_backup_dir = os.path.join(self.backup_dir, time.strftime("%Y%m%d%H", time.localtime()))
                    if not os.path.exists(sub_backup_dir):
                        os.makedirs(sub_backup_dir)
                    #备份积压文件直到积压文件数小于最大值
                    #print self.receive_file_list.size(), self.max_file_num
                    while self.receive_file_list.size() > self.max_file_num:
                        try:
                            try:
                                #获取一个待发送文件
                                filename = self.receive_file_list.get()
                                if not filename: break
                                #获取统计日志信息
                                statinfo = self.statinfo_map.find(filename)
                                #文件重命名
                                tmp_filename = filename
                                for rename in self.rename_func_list:
                                    tmp_filename = tmp_fileanme.replace(rename[0], rename[1])
                                #备份文件
                                src_file = os.path.join(self.send_dir, filename)
                                bak_file = os.path.join(sub_backup_dir, tmp_filename)
                                shutil.copy(src_file, bak_file)
                                #记录日志
                                self.error_logger.info("Success backup file: %s size: %s record: %s cause: OverStock"%(filename, statinfo.file_size, statinfo.record_num)) 
                                if self.is_write_statlog:
                                    self.statlog.write_statlog_amply(filename, statinfo, -1, self.ftpinfo.name, self.data_dest)
                            except NoKeyError, msg:
                                self.error_logger.exception("Error backup file: %s size: 0 record: 0 cause: StatLog"%(filename))
                                if self.is_write_statlog:
                                    self.statlog.write_statlog_curtly(filename, 0, "--", -1, self.ftpinfo.name, self.data_dest)
                            except Exception, msg:
                                self.error_logger.exception("Error backup file: %s size: %s record: %s cause: Unknown"%(filename, statinfo.file_size, statinfo.record_num))
                                if self.is_write_statlog:
                                    self.statlog.writr_statlog_amply(filename, statinfo, -1, self.ftpinfo.name, self.data_dest)
                        except:
                            self.error_logger.exception("unknown error")
                        self.send_file_list.add(filename)
                except:
                    self.error_logger.exception("unknown error")
                    
    def run(self):
        """
        FTP模块监视线程主体，用于检测FTP是否恢复，备份文件等
        """
        self.init_ftp_thread()
        
        while(True):
            try:
                #检测FTP服务是否恢复
                self.check_ftp()
                
                #所有FTP服务都不可用时备份文件
                self.backup_all_file()
                   
                #备份积压文件
                self.backup_stock_file()
                
            except:
                self.error_logger.exception("unknown error")
                raise   
            time.sleep(20)

import unittest
import ConfigInfo
import ConfigReader
import StatInfo
import MutexMap
  
class FTPClientTestCase(unittest.TestCase):
    def setUp(self):
        pass
        
    def tearDown(self):
        pass 
        
    def testBaseFunc(self):
        profile = ConfigReader.ConfigReader().read_profile()
        
        statlog_info = ConfigInfo.StatLog()
        statlog_info.app_name = "STATLOG_111"
        statlog_info.data_dest = "CB"
        
        ftpinfo = ConfigInfo.FTPInfo()
        ftpinfo.ip_list = ["10.0.43.80"]
        ftpinfo.port = 21
        ftpinfo.username = "fenghuo"
        ftpinfo.password = "123456"
        ftpinfo.tmp_dir = "./"
        ftpinfo.dst_dir = "./"
        ftpinfo.name = "test_ftp"
        ftpinfo.adv_conf.append(statlog_info.app_name)
                
        statinfo = StatInfo.StatInfo()
        statinfo.uparea_id = "320100"
        statinfo.protype_desc = "HTTP"
        statinfo.data_source = "111"
        statinfo.file_type = "bcp"
        statinfo.file_size = "1024"
        statinfo.record_num = "10000"
        
        filename = "MASS_1234567890_123456_HTTP_320100_111.bcp"
        fp = open(os.path.join(profile.send_dir, filename), "w")
        fp.write("ddddddddddddd")
        fp.close()
        
        statinfo_map = MutexMap.MutexMap()
        statinfo_map.insert(filename, statinfo)
        ftp_client = FTPClient(profile, [statlog_info], ftpinfo, statinfo_map)
        ftp_client.daemon = True
        ftp_client.start()
        
        ftp_client.receive_file_list.add(filename)
        time.sleep(5)
        print ", ".join(ftp_client.send_file_list.copy())
            
if __name__ == "__main__":

    suite = unittest.TestSuite()
    suite.addTest(FTPClientTestCase("testBaseFunc"))
        
    runner = unittest.TextTestRunner()
    runner.run(suite)
    